package com.toutiao.score

import scala.io.Source

import scala.collection.mutable.{HashMap => HMap}
import scala.collection.mutable.ArrayBuffer
import net.liftweb.json.{JsonAST,Extraction,Printer,parse}

import scala.math.{log,sqrt,min,max}

import org.apache.log4j.Logger

object Measure {
 
  implicit val FORMATS = net.liftweb.json.DefaultFormats

  def get(location:String) = {
    val source = Source.fromFile(location,"UTF-8")
    val lineIterator = source.getLines
    val ms = new ArrayBuffer[(String,Int)]
    for (line <- lineIterator) {
    	  val x= parse(line)
    	  val x0 = x.children(0).extract[String]
    	  val x1 = x.children(1).extract[Int]
    	  ms += ((x0,x1))
     }
    source.close
    ms.toMap    
  }
  
}

class Scorer(val M:Map[String,Int], val N:Int) {
  
  private val MIN_CC = 5
  
  private val AVGDL = 500
  private val K1 = 1.7
  private val B = 0.75
  
  private val BOOST_TITLE = 3
  
  val _log = Logger.getLogger(this.getClass.getName)
  
  def cut(text:String) = {
    text.replaceAll("\n"," ").
         replaceAll("[\\pP]", " ").
    	 replaceAll("\\p{IsZs}+", " ").toLowerCase.
    	 split(" ")
  }
  
  def getTFIDFScore(cutTitle:String,cutContent:String) = {
	val vs = cut((cutTitle+" ")*BOOST_TITLE)++cut(cutContent)
    val vm = new HMap[String,Int]
    for (v <- vs){
      val c = vm.getOrElse(v, 0)
      vm(v) = c+1
    }
    val s = vm.map(v => {
      val ss =
        if (M.contains(v._1)){
          v._2*log(N/M(v._1))
        }
        else {
          v._2*log(N/MIN_CC)
        }
      (v._1,ss)
    })
    s.toMap
  }
  
  val maxV = 100.0
  val minV = 5.0
  val mN = 30
  val X = 8.0
  
  val maxU = 0.65

  // 关键词权重不能太高
  def norm2(vs: Map[String, Double]) = {
    val us = vs.filter(_._2 > minV).toArray.
      map(vv => {
        if (vv._2 > maxV) (vv._1, maxV)
        else vv
      }).
      sortBy(-_._2)
    val cnt = if (us.size > mN) mN else us.size
    val vvs = us.slice(0, cnt).toMap
    val s = if (vvs.size < mN) {
      vvs.values.map(v => v * v).sum + (mN - vvs.size) * X * X
    } else {
      vvs.values.map(v => v * v).sum
    }
    val ss = sqrt(s)
    vvs.map(vv => {
      (vv._1, min(maxU,vv._2/ss))
    })
  }
  
  def norm1(vs:Map[String,Double]) = {
    val us = vs.toArray.sortBy(-_._2)
    val cnt = if (us.size > mN) mN else us.size
    val vvs = us.slice(0, cnt).toMap
    val s = vvs.values.map(v => v*v).sum
    val ss = sqrt(s)
    vvs.map(vv => {
      (vv._1,vv._2/ss)
    })
  }
  
  def score(cutTitle:String, cutContent:String) = {
    val vs = getTFIDFScore(cutTitle,cutContent)
    norm2(vs)
  }

}